# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from boss.items import BossItem


class BossSpiderSpider(CrawlSpider):
	name = 'boss_spider'
	allowed_domains = ['zhipin.com']
	start_urls = ['https://www.zhipin.com/c101280600/?query=python&page=2']

	rules = (
		# 获取页码url
		Rule(LinkExtractor(allow=r'/c101280600/\?query=python&page=\d'), follow=True),
		# 获取详情页数据
		Rule(LinkExtractor(allow=r'.*?job_detail/.*?\.html'), follow=False, callback='parse_item'),
	)

	def parse_item(self, response):
		dic = dict()
		dic['title'] = response.xpath("//div[@class='info-primary']/div[@class='name']/h1/text()").get()
		dic['salary'] = response.xpath("//div[@class='info-primary']/div[@class='name']/span/text()").get()
		tmp = response.xpath("//div[@class='job-primary detail-box']/div[@class='info-primary']/p/text()").getall()
		dic['city'] = tmp[0]
		dic['work_experience'] = tmp[1]
		dic['edu_cation'] = tmp[2]
		dic['company'] = response.xpath("//div[@class='detail-content']/div[@class='job-sec']/div[@class='name']/text()").get()
		dic['people_num'] = response.xpath("//div[@class='sider-company']/p[3]/text()").get()
		dic['addr'] = response.xpath("//div[@class='location-address']/text()").get()
		desc = response.xpath("//div[@class='job-sec']/div[@class='text']/text()").getall()
		dic['desc'] = ''.join(desc).replace(' ', '').replace('\r\n', '').replace('\t', '')

		item = BossItem(**dic)
		yield item